#include<iostream>
#include<fstream>
#include<vector>
#include<string>
#include<boost/filesystem.hpp>

#define Filepath "./data/input/"
const char*Outdate="./data/raw_html/raw.txt";

namespace bfs=boost::filesystem;

typedef struct page{
    std::string title;
    std::string content;
    std::string url;
}html_page;

//声明
bool ReadHtml(std::vector<std::string>* filelist);
bool parserHtml(std::vector<std::string>&filelist,std::vector<html_page>* pages);
bool SaveHtml(const std::vector<html_page>& pages);


//数据清洗
int main()
{
    std::vector<std::string> filelist;
    //读取Html文件
    ReadHtml(&filelist);
    std::cout<<"filelist:"<<filelist.size()<<std::endl;

    std::vector<html_page> pages;
    //去标签
    parserHtml(filelist,&pages);
    std::cout<<" pages:"<< pages.size()<<std::endl;

    //保存文件
    SaveHtml(pages);

    return 0;
}

//定义
//读取html文件
//读取文件时没有加目录？./date/input/boost/type_erasure/bitand_assignable.html变成./date/input/bitand_assignable.html
bool ReadHtml(std::vector<std::string>* filelist)
{
    //将strig转化为path类型
    bfs::path fp(Filepath);
    if(!bfs::exists(fp))
    {
        std::cerr<<"File noexit"<<std::endl;
    }

    //将path转化成一个迭代器方便遍历
    bfs::recursive_directory_iterator file_it(fp);
    //遍历结束条件
    bfs::recursive_directory_iterator end;
    for(;file_it!=end;file_it++)
    {
        bfs::path p=*file_it;
        //判断是否为普通文件
        //将path转为file_status，表示判断是否为普通文件的标识
        if(!bfs::is_regular_file(status(p)))
        {
            continue;
        }

        //判断是否为.html为结尾的文件
        //extension返回p的.后缀，没有则返回空
        if(p.extension()!=bfs::path(".html"))
        {
            continue;
        }
        std::string f=p.string();
        filelist->push_back(f);

        //int pos=f.rfind('/');//不能直接查找，会将之前的input到html之间的目录删掉
        //int pos=f.find(Filepath);
        //std::string fpath=f.substr(pos+sizeof(Filepath));
        //std::string fpath=f.substr(pos+1);
        //filelist->push_back(fpath);
        // filelist->push_back(p.string());
        // if(filelist->size()==1)
        // {
        //     std::cout<<fpath<<std::endl;
        // }
    }

    return true;
}

bool Htmltitle(std::string& fulltext, html_page&p){
    //查找标题
    std::string left("<title>");
    int start=fulltext.find(left);

    std::string right("</title>");
    int end=fulltext.find(right,start);
    
    //跳过<title>
    start+=left.size();
    if(start>end)
    {
        return false;
    }

    p.title=fulltext.substr(start,end-start);
    return true;
}

bool Htmlcontent(std::string& fulltext, html_page&p){
    //状态机记录
    enum {
            label=0,
            content
        };
    int flag=label;
    for(char& c:fulltext)
    {
        switch (flag)
        {
            case label:
                if(c=='>')
                {
                    flag=content;
                }
                break;
            case content:
                if(c=='<')
                {
                    flag=label;
                }
                else
                {
                    p.content+=c;
                }
                break;
            default:
                std::cerr<<"content err"<<std::endl;
                return false;
        }
    }
    return true;
}

bool Htmlurl(std::string& name, html_page& p)
{
    //url网址
    std::string url="https://www.boost.org/doc/libs/1_85_0/doc/html/";
    url+=name;
    p.url=url;
    return true;
}

bool Read(std::ifstream& f,std::string* fulltext)
{
    std::string line;
    while(getline(f,line))
    {
        *fulltext+=line;
    } 

    return true;
}


bool parserHtml(std::vector<std::string> &filelist,std::vector< html_page>* pages)
{
    int title_count=0;
    int content_count=0;
    int url_count=0;
    for(auto&name:filelist)
    {
        html_page p;

        //文件存放位置
        //std::string filename=name; 前缀问题
        //std::string filename=Filepath+name; 
        //std::cout<<filename<<std::endl;
        //std::ifstream f(filename.c_str(),std::ifstream::in);

        std::ifstream f(name.c_str(),std::ifstream::in);

        std::string fulltext; 
        Read(f,&fulltext);
        if(!Htmltitle(fulltext,p))
        {
            title_count++;
            continue;
        }

        if(!Htmlcontent(fulltext,p))
        {
            content_count++;
            continue;
        }
        //std::cout<<"content finish"<<std::endl;

        //前缀处理
        std::string pf("input/");
        int pos=name.rfind(pf);
        pos+=pf.size();
        std::string name_html=name.substr(pos);
        if(!Htmlurl(name_html,p))
        {
            url_count++;
            continue;
        }

        pages->push_back(p);
    }
    std::cout<<"title:"<<title_count<<std::endl;
    std::cout<<"content:"<<content_count<<std::endl;
    std::cout<<"url:"<<url_count<<std::endl;
    
    return true;
}


bool SaveHtml(const std::vector< html_page>& pages){
    //分割符，明确标题、内容、url的边界
    #define seg '\x03'
    //std::string o=Outdate;
    //std::ofstream Out(Outdate,std::ofstream::out,std::ofstream::binary);//binary二进制读写,ofstream只有两个参数
    //std::ofstream ofs(Outdate,std::ofstream::binary);//binary二进制读写
    std::ofstream ofs(Outdate,std::ofstream::out|std::ofstream::binary);//与操作写入多个参数

    for(auto&p:pages)
    {
        //以\n作为结尾，getline读取一行能够直接将一个文件内容全读下来
        std::string con;
        con+=p.title; 
        con+=seg;
        con+=p.content;
        con+=seg;
        con+=p.url;
        con+='\n';

        ofs.write(con.c_str(),con.size());
    }

    return true;
}

